package com.dmcb.trade.business.crawlers.author;

import com.dmcb.common.business.constants.CommonConstant;
import com.dmcb.common.business.utils.*;
import com.dmcb.common.web.conversion.JsonResult;
import com.dmcb.common.business.utils.WebClientUtil;
import com.dmcb.trade.business.constants.CrawlConstant;
import com.dmcb.trade.business.constants.TradeConstant;
import com.dmcb.trade.business.crawlers.article.BaseParser;
import com.dmcb.trade.business.crawlers.article.ToutiaoParser;
import com.dmcb.trade.business.entities.Article;
import com.dmcb.trade.business.entities.Author;
import com.dmcb.trade.business.mappers.ArticleCrawlMapper;
import com.dmcb.trade.business.services.ArticleCrawlerService;
import com.dmcb.trade.business.utils.TouTiaoCrawlUtil;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import javax.script.ScriptException;
import java.io.IOException;
import java.util.*;

/**
 * 头条爬取服务
 * Created by long on 2016/2/26.
 */
@Service
public class ToutiaoCrawler extends BaseCrawler {
    Logger logger = LoggerFactory.getLogger(ToutiaoCrawler.class);

    @Autowired
    private ToutiaoParser toutiaoParser;
    @Autowired
    private ArticleCrawlerService articleCrawlerService;
    @Autowired
    private TouTiaoSignatureUtil touTiaoSignatureUtil;

    @Autowired
    private ArticleCrawlMapper articleCrawlMapper;

    @Override
    protected String url() {
        return CrawlConstant.TOUTIAO_URL;
    }

    @Override
    protected int tag() {
        return CrawlConstant.TOUTIAO_TAG;
    }

    @Override
    @SuppressWarnings("unchecked")
    protected JsonResult crawl(int authorId, String link, Author author, Document doc, boolean all) {
        logger.info("起始连接：" + link);
        if (!link.endsWith("/") && !link.contains("mid")) {
            link = link + "/";
        }
        String as;
        int totalAll = 0;
        String id;
        if (link.contains("mid")) {
            link = link.substring(0, link.indexOf("/#mid"));
        } else {
            link = link.endsWith("/") ? link.substring(0, link.lastIndexOf("/")) : link.substring(0, link.lastIndexOf("/"));
        }
        id = link.substring(link.lastIndexOf("/") + 1).replace("m", "");

        Map<String, String> map = new HashMap<>();
        map.put("Host", "www.toutiao.com");
        map.put("Connection", "keep-alive");
        map.put("Cache-Control", "max-age=0");
        map.put("Upgrade-Insecure-Requests", "1");
        map.put("DNT", "1");
        map.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36");
        map.put("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
        map.put("Accept-Encoding", "gzip, deflate, br");
        map.put("Accept-Language", "zh-CN,zh;q=0.9");
        map.put("Cookie", "tt_webid=6563471680186336781; WEATHER_CITY=%E5%8C%97%E4%BA%AC; UM_distinctid=163ce77a337149-007eddf26a98e1-3961430f-1fa400-163ce77a33895a; tt_webid=6563471680186336781; csrftoken=7c516ddbb01b46644a4d56391dc621d5; uuid=\"w:739522d8cc694910a809147fefc28051\"; _ga=GA1.2.309194249.1529458871; ccid=cee1683522d40f1234820931f53b033b; __utma=24953151.309194249.1529458871.1540792143.1540792143.1; __utmz=24953151.1540792143.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); CNZZDATA1259612802=1531172682-1528174811-https%253A%252F%252Fwww.baidu.com%252F%7C1542155575; __tasessionId=953wbnqy71542159976268");


        List<String> urlList = new ArrayList<>();
        Map<String, Object> resultMap = new HashMap<>();
        String signature ;
        String json;
        Integer time = 0;
        List<Map<String, Object>> mapList = new ArrayList<>();
        while (true) {
            as = DateUtil.getAs();
            try {
                signature = touTiaoSignatureUtil.getSignature(id, time + "");
            } catch (IOException | ScriptException | NoSuchMethodException e) {
                e.printStackTrace();
                return error("获取参数失败，请联系客服人员");
            }
            link = "https://www.toutiao.com/c/user/article/?page_type=1&user_id=" + id + "&max_behot_time=" + time + "&count=20&as=" + as + "&cp=5B7ED0AD05E4CE1&_signature=" + signature;
            System.out.println("头条列表链接：" + link);
            for (int i = 0; i < 10; i++) {
                json = WebUtil.postRequest(link, false, map);
                resultMap = JsonUtil.getResult(json);
                mapList = (List) resultMap.get("data");
                if (!ListUtil.isEmpty(mapList)) {
                    break;
                }
                try {
                    Thread.sleep(3000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
            if (ListUtil.isEmpty(mapList)) {
                return success(CrawlConstant.MSG_WARN_NO_UPDATE);
            }
            for (Map<String, Object> dataMap : mapList) {
                urlList.add("https://www.toutiao.com" + dataMap.get("source_url"));
            }
            resultMap = (Map) resultMap.get("next");
            time = (Integer) resultMap.get("max_behot_time");
            JsonResult result = iterate2(authorId, urlList);
            if (!result.isSuccess()) {
                return result;
            }
            totalAll += result.getInt();
            //不能限制19，应该限制为1，只要此页添加成功一条都得继续下页爬取
            if (!all && result.getInt() < 1) {
                break;
            }
        }
        return totalAll == 0 ? success(CrawlConstant.MSG_WARN_NO_UPDATE) : success(totalAll);
    }


    //针对加载不了JS的头条文章获取数据的方法
    public static String getData(String start, String end, String data) {
        data = data.substring(data.indexOf(start) + start.length(), data.length());
        return data.substring(0, data.indexOf(end)).trim();
    }


    /**
     * 迭代解析、保存文章
     *
     * @return 成功保存数量
     */
    private JsonResult iterate2(int authorId, List<String> list) {
        int succ = 0;
        JsonResult result;
        try {
            WebClientUtil.wc = webClientUtil.initWc(false, false, false);
            Document doc;
            int i = 0;
            for (String link : list) {
                doc = webClientUtil.getDoc(link);
                if (doc != null && doc.select("video#vjs_video_3_html5_api").size() > 0) {
                    logger.info("该链接为视频链接,link = " + link);
                    continue;
                }
                result = crawlByNoJS(authorId, link);
                int ero = 0;
                while (result.isError()) {
                    WebClientUtil.wc = webClientUtil.initWc(true, false, true);
                    String text = webClientUtil.getDoc(link).text().trim();
                    if (text.length() < 20) {
                        RedisUtils.changeIP(15);
                        result = parse(authorId, link);
                    }
                    if (++ero > 2) {
                        break;
                    }
                }
                if (result.isError()) {
                    result = iterateOther(authorId, link);
                }
                succ += result.isSuccess() ? 1 : 0;
                if (i++ / 2 == 0)
                    JedisHelper.changeIp();
            }
        } catch (Exception e) {
            logger.error("Parser.parse error,{}", e);
            return error(CommonConstant.ABNORMAL_MSG);
        }
        return success(succ);
    }

    /**
     * 针对头条另一种格式文章做的解析
     * http://www.toutiao.com/group/6416252231113752833/#p=1
     *
     * @param authorId 用户ID
     * @param link     文章链接
     * @return 导入数据库结果
     */
    private JsonResult iterateOther(int authorId, String link) {
        Date time = null;
        StringBuilder content;
        String coverPath;
        String authorName;
        BaseParser baseParser = articleCrawlerService.parserMap.get(1);
        Author author = articleCrawlMapper.queryAuthor(authorId);
        author = getAuthor(authorId, author);
        link = "https://www.toutiao.com/" + link.substring(link.indexOf("item/") + 5) + "#p=";

        Document doc = getDocument(link + "1");
        if (doc == null) {
            return error(CommonConstant.ABNORMAL_MSG);
        }
        Article article = checkOrInit(doc, authorId, link + "1", baseParser);
        if (article == null) {
            return error(TradeConstant.MSG_ERROR_TITLE_NONE);
        }
        // 如果已经存在，直接返回
        if (article.getId() > 0) {
            return warn(article.getId(), TradeConstant.MSG_ERROR_TITLE_EXIST);
        }
        // 解析发布日期、内容、封面等
        String pageString = doc.select("span.abstract-index").text();
        if (!pageString.contains("/")) {
            logger.info("获取文章最大值失败，可能是视频连接，link = " + link + "1");
            return error("获取最大值失败");
        }
        int page = Integer.parseInt(pageString.substring(pageString.indexOf("/") + 1));
        ///group/6384671761322230018/
        String sTime = getContent("span.time", doc);
        if (StringUtils.isNotBlank(sTime)) {
            time = DateUtil.parseDate(sTime);
        }
        authorName = getContent("a.media-user>span", doc);
        coverPath = getSrc(doc);
        content = new StringBuilder(getContent("div.abstract", doc));
        if (StringUtils.isBlank(content.toString())) {
            return error("内容解析失败");
        }
        if (page > 1) {
            String src;
            for (int i = 2; i <= page; i++) {
                src = link + String.valueOf(i);
                doc = webClientUtil.getDoc(src);
                content.append(getContent("div.abstract", doc));
            }
        }
        article.setCoverPath(coverPath);
        article.setBody(content.toString());
        article.setAuthorName(authorName);
        article.setPostTime(time);
        // 设置默认值、保存
        return success(baseParser.save(author, article, "头条"));

    }


    /**
     * 针对头条另一种格式文章做的解析
     * https://www.toutiao.com/i6563146100021133828/
     * 开启JS 也没用，切割解析保存
     * 开启JS 也没用，切割解析保存
     *
     * @param authorId 用户ID
     * @param link     文章链接
     * @return 导入数据库结果
     */
    private JsonResult crawlByNoJS(int authorId, String link) {
        Date time = null;
        WebClientUtil.wc = webClientUtil.initWc(false, false, false);
        BaseParser baseParser = articleCrawlerService.parserMap.get(1);
        Author author = articleCrawlMapper.queryAuthor(authorId);
        author = getAuthor(authorId, author);
        Map<String, String> map = TouTiaoCrawlUtil.getTouTiaoInfo(link);
        if (map == null) {
            return error("解析失败，link= " + link);
        }
        String title = map.get("title");
        String sTime = map.get("time");
        String content = map.get("content");
        String authorName = map.get("authorName");
        if(StringUtils.isBlank(title)){
            return error("解析失败，未获取到标题");
        }
        // 校验该作者标题是否存在
        Article article = articleCrawlMapper.queryByAuthorTitle(authorId, title);
        if (article == null) {
            article = baseParser.initArticle(authorId, title, link);
        }
        if (article == null) {
            return error(TradeConstant.MSG_ERROR_TITLE_NONE);
        }
        // 如果已经存在，直接返回
        if (article.getId() > 0) {
            return warn(article.getId(), TradeConstant.MSG_ERROR_TITLE_EXIST);
        }

        if (StringUtils.isNotBlank(sTime)) {
            time = DateUtil.parseDate(sTime);
        }
        if (StringUtils.isBlank(content)) {
            return error("内容解析失败");
        }

        article.setBody(content);
        article.setAuthorName(authorName);
        article.setPostTime(time);
        // 设置默认值、保存
        return success(baseParser.save(author, article, "头条"));

    }

    private Author getAuthor(int authorId, Author author) {
        author = getAuthor(authorId, author, articleCrawlMapper);
        return author;
    }

    public static Author getAuthor(int authorId, Author author, ArticleCrawlMapper articleCrawlMapper) {
        if (author == null) {
            author = new Author();
            author.setAuthorId(authorId);
            author.setRank("B");
            articleCrawlMapper.insertAuthor(author);
        }
        return author;
    }


    private Document getDocument(String url) {
        HtmlPage htmlPage;
        Document doc = null;
        try {
            htmlPage = webClientUtil.initWc(true, false, true).getPage(url);
            doc = Jsoup.parse(htmlPage.asXml());
        } catch (IOException e) {
            logger.error(e.getMessage());
        }
        return doc;
    }

    private Article checkOrInit(Document doc, int authorId, String link, BaseParser baseParser) {
        String title = getContent("h2.title", doc);
        if (StringUtils.isBlank(title)) {
            logger.error("标题不存在, link={}", link);
            return null;
        }
        // 校验该作者标题是否存在
        Article article = articleCrawlMapper.queryByAuthorTitle(authorId, title);
        if (article != null) {
            return article;
        }
        return baseParser.initArticle(authorId, title, link);
    }

    /**
     * @param style    样式
     * @param document 内容
     * @return 根据样式获得的内容
     */
    public String getContent(String style, Document document) {
        Element element = document.select(style).first();
        if (element != null) {
            return element.text();
        }
        return null;
    }

    private String getSrc(Document document) {
        Element element = document.select("li.image-item>div.image-item-inner img[src]").first();
        if (element != null) {
            return element.attr("src");
        }
        return null;
    }


    @Override
    protected String avatarSelect() {
        return "a.pgc-link img[src]";
    }

    @Override
    protected String authorLink() {
        return "a.pgc-link";
    }

    @Override
    protected String channelNameSelect() {
        return "div.info-inner>p.name";
    }

    @Override
    protected String listSelect() {
        return "div#articles>ul>li";
    }

    @Override
    protected String titleSelect() {
        return "div.item-inner>a,div.article-header>h1";
    }

    @Override
    protected String authorSelect() {
        return "div.pgc-username>span";
    }

    @Override
    protected JsonResult parse(int authorId, String link) throws Exception {
        return toutiaoParser.parse(authorId, link);
    }

    @Override
    protected String getBody(Document doc) {
        return toutiaoParser.getBody(doc);
    }
}
